use lazy_static::lazy_static;
use std::collections::HashSet;
use std::ffi::OsString;
use std::fs;
use std::io::Write;
use std::path::PathBuf;

lazy_static! {
    static ref PUNCTURES: HashSet<char> =
        vec!['：', '，', '。', '“', '”', '!', '！', ';', '?', '.', ',']
            .into_iter()
            .collect();
}

#[inline]
fn is_validate(c: &char) -> bool {
    (PUNCTURES.contains(c))
        || ((*c as u32) >= 0x4e00 && (*c as u32) <= 0x9fa5)
        || *c == '\n'
        || *c == '\r'
}

fn main() -> Result<(), std::io::Error> {
    let a: Vec<(PathBuf, OsString)> = fs::read_dir(r"/root/python-projects/classical-seg/text")?
        .map(|f| f.map(|f| (f.path(), f.file_name())).unwrap())
        .collect();
    for t in a {
        let input = fs::read_to_string(t.0)?;

        let output: String = input
            .chars()
            .filter(|c| is_validate(c))
            .map(|c| if PUNCTURES.contains(&c) { ' ' } else { c })
            .collect();
        let mut output_file = fs::File::create(format!(
            "/root/python-projects/classical-seg/text_output/{}",
            t.1.to_string_lossy()
        ))?;
        write!(output_file, "{}", output)?;
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    #[test]
    fn is_validate_test() {
        use crate::is_validate;
        let test_str = "123, 啦abcd啦啦";
        let ans: String = test_str.chars().filter(|c| is_validate(c)).collect();
        assert_eq!(",啦啦啦", ans);
    }

    #[test]
    fn remove_puntures_test() {
        use crate::is_validate;
        use crate::PUNCTURES;
        let test_str = "123, 啦abcd啦啦";
        let ans: String = test_str
            .chars()
            .filter(|c| is_validate(c))
            .map(|c| if PUNCTURES.contains(&c) { ' ' } else { c })
            .collect();
        assert_eq!(" 啦啦啦", ans);
    }
}
